package com.quxing.amazon.spider;

import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.http.HttpUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.quxing.amazon.model.Category;
import com.quxing.amazon.model.Good;
import com.quxing.amazon.service.CategoryService;
import com.quxing.amazon.service.GoodService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * @author zhb
 * @description
 * @date 2021/5/9 15:53
 */
@Slf4j
@Component
@RequiredArgsConstructor
public class GoodSpider {

    private static final List<String> USER_AGENTS;
    private static final String URL_PREFIX = "https://www.amazon.cn";

    private final CategoryService categoryService;
    private final GoodService goodService;

    static {
        USER_AGENTS = ListUtil.list(false, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
                "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
                "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
                "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
                "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
                "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
                "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
                "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
                "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
                "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
                "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
                "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
                "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
                "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
                "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
                "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
                "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
                "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
                "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
                "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
                "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
                "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
                "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
                "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
                "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
                "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
                "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36");
    }

//    @Scheduled(fixedDelay = 3600000)

    /**
     * 每周1凌晨1点执行定时任务
     */
    @Scheduled(cron = "0 0 1 ? * 3")
    public void start() {
        log.info("定时任务启动！");
        List<Category> categories = categoryService.list();
        List<Integer> categoryParentIds = categories.stream()
                .map(Category::getParentId)
                .distinct()
                .collect(Collectors.toList());
        //过滤出最后一级所有分类
        List<Category> lastCategories = categories.stream()
                .filter(category -> categoryParentIds.stream()
                        .noneMatch(categoryParentId -> categoryParentId.equals(category.getId())))
                .collect(Collectors.toList());
//        List<Integer> existCategoryIds = goodService.selectCategoryIds();
//        //筛选出没有商品的分类
//        lastCategories.removeIf(category -> existCategoryIds.stream().anyMatch(categoryId -> category.getId().equals(categoryId)));
//        System.out.println(lastCategories);
        lastCategories.forEach(this::spiderByCategory);
    }

    private Integer getCategoryId() {
        return goodService.getLastOneGoodCategory();
    }

    /**
     * 根据分类爬取数据
     *
     * @param category
     */
    private void spiderByCategory(Category category) {
        String categoryLink = category.getLink();
        String urlByPage = categoryLink + "&page=1";
        Integer page = getPage(urlByPage);
        if (page >= 10) {
            page = 10;
        }
        for (Integer currentPage = 1; currentPage <= page; currentPage++) {
            List<Good> goods = new ArrayList<>();
            for (int i = 0; i < 10; i++) {
                ThreadUtil.sleep(3000);
                try {
                    goods = getGoods(category, currentPage);
                    break;
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error("获取商品数据失败:{}", e.getMessage());
                }
            }
            if (goods.size() > 0) {
                //获取所有爬取的亚马逊商品id
                List<String> amazonIds = goods.stream()
                        .map(Good::getAmazonId)
                        .collect(Collectors.toList());
                LambdaQueryWrapper<Good> queryWrapper = Wrappers.<Good>lambdaQuery().in(Good::getAmazonId, amazonIds);
                //查询数据已经存在的就商品数据
                List<Good> oldGoods = goodService.list(queryWrapper);
                Map<String, Good> oldGoodMap = oldGoods.stream()
                        .collect(Collectors.toMap(Good::getAmazonId, v -> v));
                //过滤出新爬取的旧商品数据
                List<Good> oldUpdateGoods = goods.stream()
                        .filter(good -> oldGoods.stream()
                                .anyMatch(oldGood -> oldGood.getAmazonId().equals(good.getAmazonId())))
                        .collect(Collectors.toList());
                oldUpdateGoods.forEach(good -> good.setCreateTime(oldGoodMap.get(good.getAmazonId()).getUpdateTime()).setId(oldGoodMap.get(good.getAmazonId()).getId()));
                //过滤出新爬取的新商品数据
                List<Good> newGoods = goods.stream()
                        .filter(good -> oldGoods.stream()
                                .noneMatch(oldGood -> oldGood.getAmazonId().equals(good.getAmazonId())))
                        .collect(Collectors.toList());
                if (newGoods.size() > 0) {
                    //插入新商品
                    goodService.saveBatch(newGoods, 500);
                }
                if (oldUpdateGoods.size() > 0) {
                    //更新就商品
                    goodService.updateBatchById(oldUpdateGoods);
                }

            }
        }
    }

    /**
     * 爬取指定页码分类下的商品
     *
     * @param category
     * @param currentPage
     * @return
     */
    private List<Good> getGoods(Category category, Integer currentPage) {
        String url = category.getLink() + "&page=" + currentPage;
        String body = HttpUtil.createGet(url)
                .header("User-Agent", USER_AGENTS.get(RandomUtil.randomInt(USER_AGENTS.size())))
                .execute()
                .body();
        Document document = Jsoup.parse(body);
        Elements elements = document.select("div.s-expand-height");
        log.info("当前分类:{},第{}页,爬取到商品数据:{}条", category.getName(), currentPage, elements.size());
        List<Good> goods = new ArrayList<>();
        for (Element element : elements) {
            try {
                Element img = element.selectFirst("span.rush-component img.s-image");
                String src = img.attr("src");
                Element goodElement = element.selectFirst("div.a-spacing-top-small h2 a.a-text-normal");
                String title = goodElement.text();
                if (title.length() > 250) {
                    title = title.substring(0, 250);
                }
                String href = goodElement.attr("href");
                String amazonId = href.substring(4, 14);
                String goodLink = URL_PREFIX + href;
                BigDecimal score = BigDecimal.ZERO;
                int scoreCount = 0;
                Elements spans = element.select("div.a-spacing-top-micro span");
                if (spans.toString().contains("a-declarative")) {
                    List<String> scoreInfo = spans.stream()
                            .filter(span -> span.hasAttr("aria-label"))
                            .map(span -> span.attr("aria-label"))
                            .collect(Collectors.toList());
                    if (scoreInfo.size() == 2) {
                        score = new BigDecimal(scoreInfo.get(0).substring(0, 3));
                        scoreCount = Integer.parseInt(scoreInfo.get(1).replace(",", ""));
                    }
                }
                BigDecimal price = new BigDecimal(element.selectFirst("div.a-spacing-top-small span.a-offscreen").text().replace("¥", "").replace(",", ""));
                LocalDateTime now = LocalDateTime.now();
                Good good = new Good()
                        .setCategoryId(category.getId())
                        .setImgUrl(src)
                        .setAmazonId(amazonId)
                        .setCreateTime(now)
                        .setUpdateTime(now)
                        .setLinkUrl(goodLink)
                        .setTitle(title)
                        .setPrice(price)
                        .setScore(score)
                        .setScoreCount(scoreCount);
                goods.add(good);
                log.info("爬取到商品:{}", title);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return goods;
    }

    /**
     * 获取总页码数
     *
     * @param url
     * @return
     */
    private Integer getPage(String url) {
        try {
            String body = HttpUtil.createGet(url)
                    .header("User-Agent", USER_AGENTS.get(RandomUtil.randomInt(USER_AGENTS.size())))
                    .execute()
                    .body();
            Document document = Jsoup.parse(body);
            Elements elements = document.selectFirst("ul.a-pagination").select("li");
            if (elements.size() > 2) {
                return Integer.valueOf(elements.get(elements.size() - 2).text());
            }
            return 1;
        } catch (Exception e) {
            log.error("获取页码错误:{},分类地址:{}", e.getMessage(), url);
            return 1;

        }
    }
}
